# Updated by Wei on 2014/11/09
# Updated by Wei on 2014/07/26
# plot the masterpiece

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
import matplotlib.pyplot as plt

markerList = ["v","^","<",">","d","o","x","1","2","3","4",]
colorList = ["b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w"]
linestyleStr1 = "solid"
linestyleStr2 = "dashed"
lineWidthValue = 4
qualityStandardValue = 0.6
markerListIndex = 0
markersizeValue = 12

def myfunc(x, pos=0):
    return '%1.2f'%(100*x)

def plotStuff():
    inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/quality_control_gov2_tail5K_20140711.txt"
    inputFileHandler = open(inputFileName,"r")
    
    allXValueList = []
    allYValueList = []
    
    currentXValues = []
    currentYValues = []
    currentHeaderLine = ""
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalF|dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalFromWei
        if lineElements[0].startswith("dataset"):
            # print line.strip()
            
            if currentHeaderLine != "":
                print currentHeaderLine.strip()
                for index,value in enumerate(currentXValues):
                    print currentXValues[index],currentYValues[index]
                print
                
                allXValueList.append(currentXValues)
                allYValueList.append(currentYValues)
                
            currentHeaderLine = line.strip()
            
            currentXValues = []
            currentYValues = []
            
            #dataset = lineElements[0].split("=")[0]
            #queryLength = lineElements[1].split("=")[0]
            #date = lineElements[2].split("=")[0]
            #dynamicWeight = lineElements[3].split("=")[0]
            #numOfQueries = lineElements[4].split("=")[0]
            #note = lineElements[5].split("=")[0]
        if len(lineElements) == 12 and lineElements[0].endswith("%"): # data line
            if lineElements[0][:-1] == "d":
                percentageIndexKept = 0.0008
            else:
                percentageIndexKept = float(lineElements[0][:-1]) / 100
            percentageTOP10DocumentResultPreservedAt10 = float(lineElements[-4])
            percentageTOP10PostingPreservedAt10 = float(lineElements[-3])
            percentageTOP10PostingPreservedInPrunedIndex = float(lineElements[-2])
            percentageQueryProcessingCost = float(lineElements[-1])
            currentXValues.append(percentageIndexKept)
            # currentYValues.append(percentageTOP10DocumentResultPreservedAt10)
            # currentYValues.append(percentageTOP10PostingPreservedAt10)
            # currentYValues.append(percentageTOP10PostingPreservedInPrunedIndex)
            currentYValues.append(percentageQueryProcessingCost)
    
    # final print
    print currentHeaderLine.strip()
    for index,value in enumerate(currentXValues):
        print currentXValues[index],currentYValues[index]
    print
    # blue squares: 'bs'
    # green triangle: 'g^'
    # plt.plot(currentXValues, currentYValues, 'bs')
    # plt.axis([0, 1, 0, 1])
    # plt.show()
    allXValueList.append(currentXValues)
    allYValueList.append(currentYValues)
    inputFileHandler.close()
    
    print "Overall big plot"
    print len(allXValueList)
    print len(allYValueList)
    
    xLabelStr = "% index kept" 
    # yLabelStr = "% of TOP10 document results preserved"
    # yLabelStr = "% of TOP10 postings preserved@10"
    # yLabelStr = "% of TOP10 postings preserved in pruned index"
    yLabelStr = "% of query processing cost"
    tStrP1 = xLabelStr + " " + "versus" + " " + yLabelStr 
    tStrP2 = "dataset=GOV2"
    tStrP3 = "queryLength=ALL"
    tStrP4 = "dynamicWeight=0"
    tStrP5 = "numOfQueries=5000"
    tStrComplete = tStrP1 + ". " + tStrP2 + " " + tStrP3 + " " + tStrP4 + " " + tStrP5
    plt.title(tStrComplete)
    plt.xlabel(xLabelStr)
    plt.ylabel(yLabelStr)
    plt.plot(allXValueList[0], allYValueList[0], \
             color='b', linestyle='dashed', marker='v', markerfacecolor='b', markersize=6, label='POW(pt,1)')
    plt.plot(allXValueList[1], allYValueList[1], \
             color='g', linestyle='dashed', marker='^', markerfacecolor='g', markersize=6, label='POW(pt,0.9)')
    plt.plot(allXValueList[2], allYValueList[2], \
             color='r', linestyle='dashed', marker='<', markerfacecolor='r', markersize=6, label='POW(pt,0.7)')
    plt.plot(allXValueList[3], allYValueList[3], \
             color='c', linestyle='dashed', marker='>', markerfacecolor='c', markersize=6, label='POW(pt,0.5)')
    plt.plot(allXValueList[4], allYValueList[4], \
             color='m', linestyle='dashed', marker='D', markerfacecolor='m', markersize=6, label='POW(pt,0.3)')
    plt.plot(allXValueList[5], allYValueList[5], \
             color='y', linestyle='dashed', marker='*', markerfacecolor='y', markersize=6, label='POW(pt,0.1)')
    plt.plot(allXValueList[6], allYValueList[6], \
             color='k', linestyle='dashed', marker='s', markerfacecolor='k', markersize=6, label='POW(pt,0)')
    
    '''
    plt.plot(allXValueList[0], allYValueList[0], 'bv', label='POW(pt,1)')
    plt.plot(allXValueList[1], allYValueList[1], 'g^', label='POW(pt,0.9)')
    plt.plot(allXValueList[2], allYValueList[2], 'r<', label='POW(pt,0.7)')
    plt.plot(allXValueList[3], allYValueList[3], 'c>', label='POW(pt,0.5)')
    plt.plot(allXValueList[4], allYValueList[4], 'mD', label='POW(pt,0.3)')
    plt.plot(allXValueList[5], allYValueList[5], 'y*', label='POW(pt,0.1)')
    plt.plot(allXValueList[6], allYValueList[6], 'ks', label='POW(pt,0)')
    '''
    
    plt.axis([0, 1, 0, 1])
    plt.legend(loc="upper left")
    plt.show()

def doRegression(inputFileName,dataForChartPlottingFileName):
    inputFileHandler = open(inputFileName,"r")
    outputFileHandler = open(dataForChartPlottingFileName,"w")
    
    allValueList1 = []  # for index size
    allValueList2 = []  # for top10 document results retained@10
    allValueList3 = []  # for top10 postings retained@10
    allValueList4 = []  # for top10 document results retained in pruned index
    allValueList5 = []  # for cost model1
    allValueList6 = []  # for cost model2
    allValueList7 = []  # for cost model3 (in ms)

    currentIndexSizeKeptValues = []
    currentPercentageOfTOP10DocumentResultsRetainedAt10 = []
    currentPercentageOfTOP10PostingsRetainedAt10 = []
    currentPercentageOfTOP10PostingsRetainedInPrunedIndex = []
    currentQueryProcessingCostValues1 = []
    currentQueryProcessingCostValues2 = []
    currentQueryProcessingCostValues3 = []
    currentHeaderLine = ""
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalF|dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalFromWei
        if lineElements[0].startswith("dataset"):
            # print line.strip()
            
            if currentHeaderLine != "":
                print currentHeaderLine.strip()
                for index,value in enumerate(currentIndexSizeKeptValues):
                    print currentIndexSizeKeptValues[index],currentPercentageOfTOP10DocumentResultsRetainedAt10[index],currentQueryProcessingCostValues1[index],currentQueryProcessingCostValues2[index]
                print
                
                allValueList1.append(currentIndexSizeKeptValues)
                allValueList2.append(currentPercentageOfTOP10DocumentResultsRetainedAt10)
                allValueList3.append(currentPercentageOfTOP10PostingsRetainedAt10)
                allValueList4.append(currentPercentageOfTOP10PostingsRetainedInPrunedIndex)
                allValueList5.append(currentQueryProcessingCostValues1)
                allValueList6.append(currentQueryProcessingCostValues2)
                allValueList7.append(currentQueryProcessingCostValues3)
                
            currentHeaderLine = line.strip()
            
            currentIndexSizeKeptValues = []
            currentPercentageOfTOP10DocumentResultsRetainedAt10 = []
            currentPercentageOfTOP10PostingsRetainedAt10 = []
            currentPercentageOfTOP10PostingsRetainedInPrunedIndex = []
            currentQueryProcessingCostValues1 = []
            currentQueryProcessingCostValues2 = []
            currentQueryProcessingCostValues3 = []
        
        if len(lineElements) == 16 and lineElements[0].endswith("%"): # data line
            if lineElements[0][:-1] == "d":
                percentageIndexKept = 0.0008
            else:
                percentageIndexKept = float(lineElements[0][:-1]) / 100
            
            percentageOfTOP10DocumentResultsRetainedAt10 = float(lineElements[-6])
            percentageOfTOP10PostingsRetainedAt10 = float(lineElements[-5])
            percentageOfTOP10PostingsRetainedInPrunedIndex = float(lineElements[-4])
            percentageQueryProcessingCost1 = float(lineElements[-3])
            percentageQueryProcessingCost2 = float(lineElements[-2])
            percentageQueryProcessingCost3 = float(lineElements[-1])
            
            currentIndexSizeKeptValues.append(percentageIndexKept)
            currentPercentageOfTOP10DocumentResultsRetainedAt10.append(percentageOfTOP10DocumentResultsRetainedAt10)
            currentPercentageOfTOP10PostingsRetainedAt10.append(percentageOfTOP10PostingsRetainedAt10)
            currentPercentageOfTOP10PostingsRetainedInPrunedIndex.append(percentageOfTOP10PostingsRetainedInPrunedIndex)
            currentQueryProcessingCostValues1.append(percentageQueryProcessingCost1)
            currentQueryProcessingCostValues2.append(percentageQueryProcessingCost2)
            currentQueryProcessingCostValues3.append(percentageQueryProcessingCost3)
            
    print currentHeaderLine.strip()
    for index,value in enumerate(currentIndexSizeKeptValues):
        print currentIndexSizeKeptValues[index],currentPercentageOfTOP10DocumentResultsRetainedAt10[index],currentQueryProcessingCostValues1[index],currentQueryProcessingCostValues2[index]
    print

    allValueList1.append(currentIndexSizeKeptValues)
    allValueList2.append(currentPercentageOfTOP10DocumentResultsRetainedAt10)
    allValueList3.append(currentPercentageOfTOP10PostingsRetainedAt10)
    allValueList4.append(currentPercentageOfTOP10PostingsRetainedInPrunedIndex)
    allValueList5.append(currentQueryProcessingCostValues1)
    allValueList6.append(currentQueryProcessingCostValues2)
    allValueList7.append(currentQueryProcessingCostValues3)
    
    print "len(allValueList1):",len(allValueList1)
    print "len(allValueList2):",len(allValueList2)
    print "len(allValueList3):",len(allValueList3)
    print "len(allValueList4):",len(allValueList4)
    print "len(allValueList5):",len(allValueList5)
    print "len(allValueList6):",len(allValueList6)
    print "len(allValueList7):",len(allValueList7)
    inputFileHandler.close()
    
    print "Prediction Process:"
    # do the linear regression prediction for each methods
    # 0.4 and 0.5 quality curve have been dropped
    TARGET_QUALITIES = [0.4,0.5,0.6,0.7,0.8,0.9]
    # methodNameList = ["UP","TCP","1_0","0.9_0","0.7_0","0.5_0","0.3_0","0.1_0","0.0_0"]
    methodNameList = ["1_5","0.9_5","0.7_5","0.5_5","0.3_5","0.1_5","0.0_5"]
    for methodIdentifier in range(0,len(allValueList1)):
        for i in range(0,3):
            currentPercentageOfTOP10DocumentResultsRetainedAt10 = []
            # option1: % of top10 document results retained@10
            # currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList2[methodIdentifier]
            # option2: % of top10 postings retained@10
            # currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList3[methodIdentifier]
            # option3: % of top10 postings retained in pruned index
            # currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList4[methodIdentifier]
            if i == 0:
                #stdStr = "quality standard: TOP10 docs retained@10"
                #print stdStr
                #outputFileHandler.write(stdStr + "\n")
                currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList2[methodIdentifier]
            if i == 1:
                #stdStr = "quality standard: TOP10 postings retained@10"
                #print stdStr
                #outputFileHandler.write(stdStr + "\n")
                currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList3[methodIdentifier]
            if i == 2:
                #stdStr = "quality standard: TOP10 postings retained in pruned index"
                #print strStr
                #outputFileHandler.write(stdStr + "\n")
                currentPercentageOfTOP10DocumentResultsRetainedAt10 = allValueList4[methodIdentifier]
            
            currentIndexSizeKeptValues = allValueList1[methodIdentifier]
            currentQueryProcessingCostValues1 = allValueList5[methodIdentifier]
            currentQueryProcessingCostValues2 = allValueList6[methodIdentifier]
            currentQueryProcessingCostValues3 = allValueList7[methodIdentifier]
            
            for index in range(0,len(currentIndexSizeKeptValues)-1):        
                # 6 points ready
                x1 = float(currentPercentageOfTOP10DocumentResultsRetainedAt10[index])
                x2 = float(currentPercentageOfTOP10DocumentResultsRetainedAt10[index+1])
                
                y11 = float(currentIndexSizeKeptValues[index])
                y12 = float(currentIndexSizeKeptValues[index+1])
    
                # Updated by Wei on 2014/07/22
                # sum cost model: currentQueryProcessingCostValues1
                # log cost model: currentQueryProcessingCostValues2
                y21 = float(currentQueryProcessingCostValues1[index])
                y22 = float(currentQueryProcessingCostValues1[index+1])
                
                y31 = float(currentQueryProcessingCostValues2[index])
                y32 = float(currentQueryProcessingCostValues2[index+1])
                
                y41 = float(currentQueryProcessingCostValues3[index])
                y42 = float(currentQueryProcessingCostValues3[index+1])
                
                
                xi = np.array([x1,x2])
                A = np.array([ xi, np.ones(2)])
                
                # linearly generated sequence
                yIndexSizeKeptList = [y11, y12]
                weightsSet1 = np.linalg.lstsq(A.T,yIndexSizeKeptList)[0] # obtaining the parameters
        
                # linearly generated sequence
                yQueryProcessingCostList1 = [y21,y22]
                weightsSet2 = np.linalg.lstsq(A.T,yQueryProcessingCostList1)[0] # obtaining the parameters
    
                # linearly generated sequence
                yQueryProcessingCostList2 = [y31,y32]
                weightsSet3 = np.linalg.lstsq(A.T,yQueryProcessingCostList2)[0] # obtaining the parameters            
                
                # linearly generated sequence
                yQueryProcessingCostList3 = [y41,y42]
                weightsSet4 = np.linalg.lstsq(A.T,yQueryProcessingCostList3)[0] # obtaining the parameters                
                
                # print currentIndexSizeKeptValues[index],currentPercentageOfTOP10DocumentResultsRetainedAt10[index],currentQueryProcessingCostValues2[index]
                # print currentIndexSizeKeptValues[index+1],currentPercentageOfTOP10DocumentResultsRetainedAt10[index+1],currentQueryProcessingCostValues2[index+1]
                
                #print "regression:"
                #print "x1:",x1,"y11:",y11,"y12:",y12
                #print "x2:",x2,"y21:",y21,"y22:",y22
                #print "m1:",weightsSet1[0],"c1:",weightsSet1[1]
                #print "m2:",weightsSet2[0],"c2:",weightsSet2[1]
                # print "prediction"
                for target_quality in TARGET_QUALITIES:
                    if target_quality >= x1 and target_quality < x2:
                        target_indexSizeKept = target_quality * weightsSet1[0] + weightsSet1[1]
                        target_queryProcessingCost1 = target_quality * weightsSet2[0] + weightsSet2[1]
                        target_queryProcessingCost2 = target_quality * weightsSet3[0] + weightsSet3[1]
                        target_queryProcessingCost3 = target_quality * weightsSet4[0] + weightsSet4[1]
                        outputFileLine = str(target_quality) + " " + str(target_indexSizeKept) + " " + str(target_queryProcessingCost1) + " " + str(target_queryProcessingCost2) + " " + str(target_queryProcessingCost3) + " " + str(methodNameList[methodIdentifier]) + " " + str(i)
                        print outputFileLine
                        outputFileHandler.write(outputFileLine + "\n")
                # print
            print
    print
    print "Overall:"
    print "inputFile",inputFileName
    print "outputFile",dataForChartPlottingFileName
    inputFileHandler.close()
    outputFileHandler.close()
    
def doRegressionExample():
    xi = np.arange(0,9)
    print xi
    print type(xi)
    exit(1)
    A = np.array([ xi, np.ones(9)])
    print A
    # linearly generated sequence
    y = [19, 20, 20.5, 21.5, 22, 23, 23, 25.5, 24]
    print A.T
    w = np.linalg.lstsq(A.T,y)[0] # obtaining the parameters
    print "m:",w[0],"c:",w[1]
    # plotting the line
    line = w[0]*xi+w[1] # regression line
    plt.plot(xi,line,'r-',xi,y,'o')
    plt.show()

def subplotForGOV2Tail5K_QPC_indexSize_tradeoff(dataForChartPlottingFileName,qualityStandardValue):
    ifh = open(dataForChartPlottingFileName,"r")  

    xValuesQuality1 = []
    yValuesQuality1 = []
    anotationQuality1 = []
    
    xValuesQuality2 = []
    yValuesQuality2 = []
    anotationQuality2 = []
    
    xValuesQuality3 = []
    yValuesQuality3 = []
    anotationQuality3 = []
    
    xValuesQuality4 = []
    yValuesQuality4 = []
    anotationQuality4 = []
    
    xValuesQuality5 = []
    yValuesQuality5 = []
    anotationQuality5 = []
    
    xValuesQuality6 = []
    yValuesQuality6 = []
    anotationQuality6 = []
    
    for line in ifh.readlines():
        lineElements = line.strip().split(" ")
        qualityStr = lineElements[0]
        indexKept = float(lineElements[1])
        QPCCost1 = float(lineElements[2])
        QPCCost2 = float(lineElements[3])
        methodAnotation = lineElements[4]
        if qualityStandardValue == int(lineElements[5]):
            if qualityStr == "0.4":
                xValuesQuality1.append(indexKept)
                yValuesQuality1.append(QPCCost1)
                anotationQuality1.append(methodAnotation)        
            
            if qualityStr == "0.5":
                xValuesQuality2.append(indexKept)
                yValuesQuality2.append(QPCCost1)
                anotationQuality2.append(methodAnotation)
            
            if qualityStr == "0.6":
                xValuesQuality3.append(indexKept)
                yValuesQuality3.append(QPCCost1)
                anotationQuality3.append(methodAnotation)
            
            if qualityStr == "0.7":
                xValuesQuality4.append(indexKept)
                yValuesQuality4.append(QPCCost1)
                anotationQuality4.append(methodAnotation)
                
            if qualityStr == "0.8":
                xValuesQuality5.append(indexKept)
                yValuesQuality5.append(QPCCost1)
                anotationQuality5.append(methodAnotation)
                
            if qualityStr == "0.9":
                xValuesQuality6.append(indexKept)
                yValuesQuality6.append(QPCCost1)
                anotationQuality6.append(methodAnotation)
    labelStr = ""
    if qualityStandardValue == 0:
        labelStr = "TOP10 docs retained@10"
    elif qualityStandardValue == 1:
        labelStr = "TOP10 postings retained@10"
    elif qualityStandardValue == 2:
        labelStr = "TOP10 postings retained in pruned index"
    
    xAxisLabel = "index kept(% of postings kept)"
    yAxisLabel = "QPC(% of postings evaluated)"
    labelStr2 = "quality curve"
    
    tStrP1 = "% index size VS. % query processing cost" 
    tStrP2 = "dataset=GOV2"
    tStrP3 = "queryLength=ALL"
    tStrP4 = "dynamicWeight=0"
    tStrP5 = "numOfQueries=5000"
    tStrComplete = tStrP1 + ". " + tStrP2 + " " + tStrP3 + " " + tStrP4 + " " + tStrP5
    
    anotations = ["1.0","0.9","0.5","0.3","0.1","0.0"]
    linestyleStr = "solid"
    lineWidthValue = 4
    plt.figure(1)
    # plt.title(tStrComplete)
    plt.subplot(321)
    plt.plot(xValuesQuality1, yValuesQuality1, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.4" + " " + labelStr2)
    
    for i in range(0,len(xValuesQuality1)):
        plt.annotate(anotations[i], xy=(xValuesQuality1[i], yValuesQuality1[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
    
    plt.subplot(322)
    plt.plot(xValuesQuality2, yValuesQuality2, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.5" + " " + labelStr2)
    for i in range(0,len(xValuesQuality2)):
        plt.annotate(anotations[i], xy=(xValuesQuality2[i], yValuesQuality2[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
    
    plt.subplot(323)
    plt.plot(xValuesQuality3, yValuesQuality3, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.6" + " " + labelStr2)
    for i in range(0,len(xValuesQuality3)):
        plt.annotate(anotations[i], xy=(xValuesQuality3[i], yValuesQuality3[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
    
    plt.subplot(324)
    plt.plot(xValuesQuality4, yValuesQuality4, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.7" + " " + labelStr2)
    for i in range(0,len(xValuesQuality4)):
        plt.annotate(anotations[i], xy=(xValuesQuality4[i], yValuesQuality4[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
        
    plt.subplot(325)
    plt.plot(xValuesQuality5, yValuesQuality5, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.8" + " " + labelStr2)
    for i in range(0,len(xValuesQuality5)):
        plt.annotate(anotations[i], xy=(xValuesQuality5[i], yValuesQuality5[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
    
    plt.subplot(326)
    plt.plot(xValuesQuality6, yValuesQuality6, \
             color='b', linestyle=linestyleStr, linewidth=lineWidthValue, marker='D', markerfacecolor='r', markersize=6, label = "0.9" + " " + labelStr2)
    for i in range(0,len(xValuesQuality6)):
        plt.annotate(anotations[i], xy=(xValuesQuality6[i], yValuesQuality6[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.grid(True)
    plt.ylabel(yAxisLabel)
    plt.xlabel(xAxisLabel)
    
    plt.show()
    ifh.close()

# don't think we need to use it, but just as a backup
def subplotForGOV2150HJQueries():
    # for gov2 150HJ queries
    # "rawDataForQualityControlChartPlotting_20140712_GOV2_150HJQueries.txt"
    ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawDataForQualityControlChartPlotting_20140712_GOV2_150HJQueries_COMPLETED.txt"
    ifh = open(ifn,"r")  

    xValuesForPoint3Quality = []
    yValuesForPoint3Quality = []
    anotationForPoint3Quality = []

    xValuesForPoint35Quality = []
    yValuesForPoint35Quality = []
    anotationForPoint35Quality = []
        
    xValuesForPoint4Quality = []
    yValuesForPoint4Quality = []
    anotationForPoint4Quality = []
    
    xValuesForPoint45Quality = []
    yValuesForPoint45Quality = []
    anotationForPoint45Quality = []
    
    for line in ifh.readlines():
        lineElements = line.strip().split(" ")
        qualityStr = lineElements[0]
        indexKept = float(lineElements[1])
        QPCCost = float(lineElements[2])
        methodAnotation = lineElements[3]

        if qualityStr == "0.3":
            xValuesForPoint3Quality.append(indexKept)
            yValuesForPoint3Quality.append(QPCCost)
            anotationForPoint3Quality.append(methodAnotation)        
        
        if qualityStr == "0.35":
            xValuesForPoint35Quality.append(indexKept)
            yValuesForPoint35Quality.append(QPCCost)
            anotationForPoint35Quality.append(methodAnotation)
        
        if qualityStr == "0.4":
            xValuesForPoint4Quality.append(indexKept)
            yValuesForPoint4Quality.append(QPCCost)
            anotationForPoint4Quality.append(methodAnotation)
        
        if qualityStr == "0.45":
            xValuesForPoint45Quality.append(indexKept)
            yValuesForPoint45Quality.append(QPCCost)
            anotationForPoint45Quality.append(methodAnotation)
    
    tStrP1 = "% index size VS. % query processing cost" 
    tStrP2 = "dataset=GOV2"
    tStrP3 = "queryLength=ALL"
    tStrP5 = "numOfQueries=150"
    tStrComplete = tStrP1 + ". " + tStrP2 + " " + tStrP3 + " " + tStrP5
    plt.title(tStrComplete)
    plt.figure(1)
    plt.subplot(321)
    plt.plot(xValuesForPoint3Quality, yValuesForPoint3Quality, \
             color='b', linestyle='dashed', marker='D', markerfacecolor='b', markersize=6, label = "P@10 0.3")
    for i in range(0,len(xValuesForPoint3Quality)):
        plt.annotate(anotationForPoint3Quality[i], xy=(xValuesForPoint3Quality[i], yValuesForPoint3Quality[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.ylabel("QPC")
    plt.xlabel("index kept")
    
    plt.subplot(222)
    plt.plot(xValuesForPoint35Quality, yValuesForPoint35Quality, \
             color='g', linestyle='dashed', marker='D', markerfacecolor='g', markersize=6, label = "P@10 0.35")
    for i in range(0,len(xValuesForPoint35Quality)):
        plt.annotate(anotationForPoint35Quality[i], xy=(xValuesForPoint35Quality[i], yValuesForPoint35Quality[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.ylabel("QPC")
    plt.xlabel("index kept")
        
    plt.subplot(223)
    plt.plot(xValuesForPoint4Quality, yValuesForPoint4Quality, \
             color='b', linestyle='dashed', marker='D', markerfacecolor='b', markersize=6, label = "P@10 0.4")
    for i in range(0,len(xValuesForPoint4Quality)):
        plt.annotate(anotationForPoint4Quality[i], xy=(xValuesForPoint4Quality[i], yValuesForPoint4Quality[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.ylabel("QPC")
    plt.xlabel("index kept")
        
    plt.subplot(224)
    plt.plot(xValuesForPoint45Quality, yValuesForPoint45Quality, \
             color='r', linestyle='dashed', marker='D', markerfacecolor='r', markersize=6, label = "P@10 0.45")
    for i in range(0,len(xValuesForPoint45Quality)):
        plt.annotate(anotationForPoint45Quality[i], xy=(xValuesForPoint45Quality[i], yValuesForPoint45Quality[i]), xytext=(-20,20), 
                    textcoords='offset points', ha='center', va='bottom',
                    bbox=dict(boxstyle='round,pad=0.2', fc='yellow', alpha=0.3),
                    arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0.5', color='red'))
    plt.legend()
    plt.ylabel("QPC")
    plt.xlabel("index kept")

    plt.show()
    ifh.close()

def oneBigPlotForGOV2Tail5K_QPC_indexSize_tradeoff(dataForChartPlottingFileName,qualityStandardValue,costModelValue):
    # text for the plot
    # qualityStandardValue,costModelValue

    TITLE_STR_PART1 = "solid lines: unigram - 0, dashed lines: unigram - 5, 'x':TCP, '+':UP. The lines from top to bottom are: 0.9/0.8/0.7/0.6 quality curves for "
    TITLE_STR_PART2 = "N/A"
    x_AXIS_LABEL = "index kept"
    y_AXIS_LABEL = "N/A"
    if costModelValue == 0:
        y_AXIS_LABEL = "QPC_SUM"
    elif costModelValue == 1:
        y_AXIS_LABEL = "QPC_LOG"
    elif costModelValue == 2:
        y_AXIS_LABEL = "QPC_ms"
    
    if qualityStandardValue == 0:
        TITLE_STR_PART2 = "resultKept"
    elif qualityStandardValue == 2:
        TITLE_STR_PART2 = "postingKept"
    
    OFN1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/QPC_IndexSize_tradeoff_v10_20140725_" + y_AXIS_LABEL + "_" + TITLE_STR_PART2 +".eps"
    OFN2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/QPC_IndexSize_tradeoff_v10_20140725_" + y_AXIS_LABEL + "_" + TITLE_STR_PART2 + ".png"
    
    TITLE_STR = ""
    # TITLE_STR = TITLE_STR_PART1 + TITLE_STR_PART2
    
    ifh = open(dataForChartPlottingFileName,"r")  

    xValuesQualityList = []
    yValuesQualityList = []

    xValuesQuality1 = []
    yValuesQuality1 = []
    anotationQuality1 = []
    
    xValuesQuality2 = []
    yValuesQuality2 = []
    anotationQuality2 = []
    
    xValuesQuality3 = []
    yValuesQuality3 = []
    anotationQuality3 = []
    
    xValuesQuality4 = []
    yValuesQuality4 = []
    anotationQuality4 = []
    
    xValuesQuality5 = []
    yValuesQuality5 = []
    anotationQuality5 = []
    
    xValuesQuality6 = []
    yValuesQuality6 = []
    anotationQuality6 = []
    
    for line in ifh.readlines():
        lineElements = line.strip().split(" ")
        qualityStr = lineElements[0]
        indexKept = float(lineElements[1])
        QPCCost1 = float(lineElements[2])
        QPCCost2 = float(lineElements[3])
        QPCCost3 = float(lineElements[4])
        methodAnotation = lineElements[5]
        if qualityStandardValue == int(lineElements[6]):
            '''
            if qualityStr == "0.4":
                xValuesQuality1.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality1.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality1.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality1.append(QPCCost3)
                anotationQuality1.append(methodAnotation)        
            '''
            if qualityStr == "0.5":
                xValuesQuality2.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality2.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality2.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality2.append(QPCCost3)
                anotationQuality2.append(methodAnotation)
            
            if qualityStr == "0.6":
                xValuesQuality3.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality3.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality3.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality3.append(QPCCost3)
                anotationQuality3.append(methodAnotation)
            
            if qualityStr == "0.7":
                xValuesQuality4.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality4.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality4.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality4.append(QPCCost3)
                anotationQuality4.append(methodAnotation)
                
            if qualityStr == "0.8":
                xValuesQuality5.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality5.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality5.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality5.append(QPCCost3)
                anotationQuality5.append(methodAnotation)
                
            if qualityStr == "0.9":
                xValuesQuality6.append(indexKept)
                if costModelValue == 0:
                    yValuesQuality6.append(QPCCost1)
                elif costModelValue == 1:
                    yValuesQuality6.append(QPCCost2)
                elif costModelValue == 2:
                    yValuesQuality6.append(QPCCost3)
                anotationQuality6.append(methodAnotation)
    
    print "len(xValuesQuality1):",len(xValuesQuality1)
    print "len(xValuesQuality2):",len(xValuesQuality2)
    print "len(xValuesQuality3):",len(xValuesQuality3)
    print "len(xValuesQuality4):",len(xValuesQuality4)
    print "len(xValuesQuality5):",len(xValuesQuality5)
    print "len(xValuesQuality6):",len(xValuesQuality6)
    print "len(yValuesQuality1):",len(yValuesQuality1)
    print "len(yValuesQuality2):",len(yValuesQuality2)
    print "len(yValuesQuality3):",len(yValuesQuality3)
    print "len(yValuesQuality4):",len(yValuesQuality4)
    print "len(yValuesQuality5):",len(yValuesQuality5)
    print "len(yValuesQuality6):",len(yValuesQuality6)
    # exit(1)
    anotations = ["1.0_5","0.9_5","0.7_5","0.5_5","0.3_5","0.1_5","0.0_5"]
    # anotations = ["UP","TCP","1.0_5","0.7_5","0.9_5","0.5_5","0.3_5","0.1_5","0.0_5","1.0_0","0.7_0","0.9_0","0.5_0","0.3_0","0.1_0","0.0_0"]
    linestyleStr1 = "solid"
    linestyleStr2 = "dashed"
    lineWidthValue = 4
    qualityStandardValue = 0.6
    markerListIndex = 0
    # markersizeValue = 15
        
    xValuesQualityList.append(xValuesQuality1)
    xValuesQualityList.append(xValuesQuality2)
    xValuesQualityList.append(xValuesQuality3)
    xValuesQualityList.append(xValuesQuality4)
    xValuesQualityList.append(xValuesQuality5)
    xValuesQualityList.append(xValuesQuality6)
    
    yValuesQualityList.append(yValuesQuality1)
    yValuesQualityList.append(yValuesQuality2)
    yValuesQualityList.append(yValuesQuality3)
    yValuesQualityList.append(yValuesQuality4)
    yValuesQualityList.append(yValuesQuality5)
    yValuesQualityList.append(yValuesQuality6)
    

    for index,currentXValuesQuality in enumerate(xValuesQualityList):
        currentYValuesQuality = yValuesQualityList[index]
        '''
        # plot the TCP
        plt.plot(currentXValuesQuality[0], currentYValuesQuality[0], \
                 colorList[index] + "x", markersize=20)
        
        # plot the UP
        plt.plot(currentXValuesQuality[1], currentYValuesQuality[1], \
                 colorList[index] + '+', markersize=20)
        '''
                
        '''
        # plot the quality curve for dynamic weight == 0
        plt.plot(currentXValuesQuality[2:9], currentYValuesQuality[2:9], \
             color=colorList[index], linestyle=linestyleStr1, linewidth=lineWidthValue, marker="o", markerfacecolor='k', markersize=10)
        
        # plot the quality curve for dynamic weight == 5
        plt.plot(currentXValuesQuality[9:15], currentYValuesQuality[9:15], \
             color=colorList[index], linestyle=linestyleStr2, linewidth=lineWidthValue, marker="o", markerfacecolor='k', markersize=10)
        '''
        
        # plot the quality curve for dynamic weight == 5
        plt.plot(currentXValuesQuality, currentYValuesQuality, \
             color=colorList[index], linestyle=linestyleStr2, linewidth=lineWidthValue, marker="o", markerfacecolor='k', markersize=10)
        qualityStandardValue += 0.1
        markerListIndex += 1  
    
    plt.title(TITLE_STR)
    # plt.legend(loc="upper left")
    plt.grid(True)
    plt.ylabel(y_AXIS_LABEL)
    plt.xlabel(x_AXIS_LABEL)
    # plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.1)
    # fig.tight_layout()
    # save it to my computer
    
    #fig = matplotlib.pyplot.gcf()
    #fig.set_size_inches(18.5,10.5)
    #fig.savefig('test2png.png',dpi=100)
    
    plt.rcParams.update({'font.size': 18})
    
    fig = plt.gcf()
    fig.set_size_inches(18.5,12.5)
    fig.savefig(OFN1,format='eps', dpi=1000, papertype="letter", bbox_inches='tight')
    fig.savefig(OFN2,format='png', papertype="letter", bbox_inches='tight')
    
    print "OFN1:",OFN1
    print "OFN2:",OFN2
    plt.show()
    ifh.close()

def plotBigrams(inputFileName):
    inputFileHandler = open(inputFileName,"r")
    
    allXValueList = []
    allYValueList = []
    
    currentXValues = []
    currentYValues = []
    currentHeaderLine = ""
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalF|dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalFromWei
        if lineElements[0].startswith("dataset"):
            # print line.strip()
            
            if currentHeaderLine != "":
                print currentHeaderLine.strip()
                for index,value in enumerate(currentXValues):
                    print currentXValues[index],currentYValues[index]
                print
                
                allXValueList.append(currentXValues)
                allYValueList.append(currentYValues)
                
            currentHeaderLine = line.strip()
            
            currentXValues = []
            currentYValues = []
        
        if len(lineElements) == 15 and lineElements[0].endswith("%"): # data line
            if lineElements[0][:-1] == "d":
                percentageIndexKept = 0.0008
            else:
                percentageIndexKept = float(lineElements[0][:-1]) / 100
            percentageTOP10DocumentResultPreservedAt10 = float(lineElements[-5])
            percentageTOP10PostingPreservedAt10 = float(lineElements[-4])
            percentageTOP10PostingPreservedInPrunedIndex = float(lineElements[-3])
            #percentageQueryProcessingCost = float(lineElements[-2])
            currentXValues.append(percentageIndexKept)
            # currentYValues.append(percentageTOP10DocumentResultPreservedAt10)
            # currentYValues.append(percentageTOP10PostingPreservedAt10)
            currentYValues.append(percentageTOP10PostingPreservedInPrunedIndex)
            # currentYValues.append(percentageQueryProcessingCost)
    
    # final print
    print currentHeaderLine.strip()
    for index,value in enumerate(currentXValues):
        print currentXValues[index],currentYValues[index]
    print
    
    print "currentXValues:",currentXValues
    print "currentYValues:",currentYValues
    allXValueList.append(currentXValues)
    allYValueList.append(currentYValues)
    inputFileHandler.close()
    
    print "Overall big plot"
    print len(allXValueList)
    print len(allYValueList)
    #exit(1)
    
    # exit(1)
    xLabelStr = "index kept" 
    # yLabelStr = "TOP10 docs retained"
    # yLabelStr = "TOP10 postings retained@10"
    yLabelStr = "TOP10 postings retained@pruned index"
    tStrP1 = xLabelStr + " " + "versus" + " " + yLabelStr 
    # tStrP2 = "dataset=GOV2, model=unigram"
    tStrP2 = "dataset=GOV2, model=Unigram & Bigram"
    tStrComplete = tStrP1 + ". " + tStrP2
    plt.title(tStrComplete)
    plt.xlabel(xLabelStr)
    plt.ylabel(yLabelStr)
    colorList = ["b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w"]
    markerList = ["v","^","<",">","D","*","s","p",'*','h','H','+','x','D','d','|','_',"v","^","<",">","D","*","s","p",'*','h','H','+','x','D','d','|','_']
    tempList1 = ["dynamic weight == 0", "dynamic weight == 5","dynamic weight == 10"]
    tempList2 = ["dynamic weight == 0", "dynamic weight == 3","dynamic weight == 5"]
    tempList3 = ["unigram static - 0","unigram dynamic - 5","bigrams-0.78-50-qb.dyn","bigrams-0.78-qb.static","bigrams-0.78-qb.dyn","bigrams-1-qb.static","bigrams-1-qb.dyn","bigrams-0.75-qb-gov.static","bigrams-0.75-qb-gov.dyn","bigrams-0.54-qb-static","bigrams-0.54-qb.dyn"]
    tempList4 = ["0 - ALL","0 - 1","0 - 2","0 - 3","0 - 4","0 - 5","0 - 6","0 - 7","0 - 8","0 - 9","0 - 10","10 - ALL","10 - 1","10 - 2","10 - 3","10 - 4","10 - 5","10 - 6","10 - 7","10 - 8","10 - 9","10 - 10"]
    print "len(colorList):",len(colorList)
    print "len(markerList):",len(markerList)
    print "len(tempList1):",len(tempList1)
    print "len(tempList2):",len(tempList2)
    print "len(tempList3):",len(tempList3)
    print "len(tempList4):",len(tempList4)
    # exit(1)
    
    labelList = tempList3
    for i in range(0,len(allXValueList)):
        plt.plot(allXValueList[i][1:], allYValueList[i][1:], \
                 color=colorList[i], linestyle='dashed', marker=markerList[i], markerfacecolor='b', markersize=6, label=labelList[i])    
    
    plt.axis([0.0, 0.1, 0.0, 1.0]) 
    plt.grid(True)
    plt.legend(loc="lower right")
    plt.show()

def plotVaryingDynamicWeight(inputFileName,qualityStandardValue):
    if qualityStandardValue == 0:
        y_AXIS_LABEL = "resultKept"
    elif qualityStandardValue == 2:
        y_AXIS_LABEL = "postingKept"
    else:
        print "sth wrong."
        exit(1)
    
    OFN1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/varying_alpha_v10_20140725_" + y_AXIS_LABEL +".eps"
    OFN2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/varying_alpha_v10_20140725_" + y_AXIS_LABEL + ".png"
    
    inputFileHandler = open(inputFileName,"r")
    
    allXValueList = []
    allYValueList = []
    
    currentXValues = []
    currentYValues = []
    currentHeaderLine = ""
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalF|dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalFromWei
        if lineElements[0].startswith("dataset"):
            # print line.strip()
            
            if currentHeaderLine != "":
                print currentHeaderLine.strip()
                for index,value in enumerate(currentXValues):
                    print currentXValues[index],currentYValues[index]
                print
                
                allXValueList.append(currentXValues)
                allYValueList.append(currentYValues)
                
            currentHeaderLine = line.strip()
            
            currentXValues = []
            currentYValues = []
        
        if len(lineElements) == 15 and lineElements[0].endswith("%"): # data line
            if lineElements[0][:-1] == "d":
                percentageIndexKept = 0.0008
            else:
                percentageIndexKept = float(lineElements[0][:-1]) / 100
            percentageTOP10DocumentResultPreservedAt10 = float(lineElements[-5])
            percentageTOP10PostingPreservedAt10 = float(lineElements[-4])
            percentageTOP10PostingPreservedInPrunedIndex = float(lineElements[-3])
            #percentageQueryProcessingCost = float(lineElements[-2])
            currentXValues.append(percentageIndexKept)
            if qualityStandardValue == 0:
                currentYValues.append(percentageTOP10DocumentResultPreservedAt10)
            elif qualityStandardValue == 2:
                currentYValues.append(percentageTOP10PostingPreservedInPrunedIndex)
            # currentYValues.append(percentageQueryProcessingCost)
    
    # final print
    print currentHeaderLine.strip()
    for index,value in enumerate(currentXValues):
        print currentXValues[index],currentYValues[index]
    print
    
    print "currentXValues:",currentXValues
    print "currentYValues:",currentYValues
    allXValueList.append(currentXValues)
    allYValueList.append(currentYValues)
    inputFileHandler.close()
    
    print "Overall big plot"
    print len(allXValueList)
    print len(allYValueList)
    # exit(1)
    
    # exit(1)
    xLabelStr = "index kept"
    plt.xlabel(xLabelStr)
    plt.ylabel(y_AXIS_LABEL)
    colorList = ["b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w","b","g","r","c","m","y","k","w"]
    markerList = ["v","^","<",">","D","*","s","p",'*','h','H','+','x','D','d','|','_',"v","^","<",">","D","*","s","p",'*','h','H','+','x','D','d','|','_']
    tempList1 = ["10","5","0"]
    tempList2 = ["dynamic weight == 0", "dynamic weight == 3","dynamic weight == 5"]
    
    print "len(colorList):",len(colorList)
    print "len(markerList):",len(markerList)
    print "len(tempList1):",len(tempList1)
    print "len(tempList2):",len(tempList2)
    

    
    
    labelList = tempList1
    for i in range(0,len(allXValueList)):
        plt.plot(allXValueList[i][1:11], allYValueList[i][1:11], \
                 color=colorList[i], linestyle=linestyleStr1, marker=markerList[i], markerfacecolor='b', markersize=6, label=labelList[i])    
    
    plt.axis([0.0, 0.1, 0.4, 0.8]) 
    plt.grid(True)
    plt.legend(loc="upper left")

    plt.rcParams.update({'font.size': 18})
    
    fig = plt.gcf()
    fig.set_size_inches(18.5,12.5)
    fig.savefig(OFN1,format='eps', dpi=1000, papertype="letter", bbox_inches='tight')
    fig.savefig(OFN2,format='png', papertype="letter", bbox_inches='tight')
    
    print "OFN1:",OFN1
    print "OFN2:",OFN2
    
    plt.show()

def plotBreakingDownQueryLength(inputFileName,qualityStandardValue):
    if qualityStandardValue == 0:
        y_AXIS_LABEL = "resultKept"
    elif qualityStandardValue == 2:
        y_AXIS_LABEL = "postingKept"
    else:
        print "sth wrong."
        exit(1)
    
    OFN1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/breakingDown_queryLength_v10_20140725_" + y_AXIS_LABEL +".eps"
    OFN2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/WSDM2015_Paper/breakingDown_queryLength_v10_20140725_" + y_AXIS_LABEL + ".png"
    
    inputFileHandler = open(inputFileName,"r")
    
    allXValueList = []
    allYValueList = []
    
    currentXValues = []
    currentYValues = []
    currentHeaderLine = ""
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        # dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalF|dataset=GOV2 queryLength=ALL date=20140708 dynamicWeight=0 numOfQueries=5000 note=ptPowTo1_ptOriginalFromWei
        if lineElements[0].startswith("dataset"):
            # print line.strip()
            
            if currentHeaderLine != "":
                print currentHeaderLine.strip()
                for index,value in enumerate(currentXValues):
                    print currentXValues[index],currentYValues[index]
                print
                
                allXValueList.append(currentXValues)
                allYValueList.append(currentYValues)
                
            currentHeaderLine = line.strip()
            
            currentXValues = []
            currentYValues = []
        
        if len(lineElements) == 15 and lineElements[0].endswith("%"): # data line
            if lineElements[0][:-1] == "d":
                percentageIndexKept = 0.0008
            else:
                percentageIndexKept = float(lineElements[0][:-1]) / 100
            percentageTOP10DocumentResultPreservedAt10 = float(lineElements[-5])
            percentageTOP10PostingPreservedAt10 = float(lineElements[-4])
            percentageTOP10PostingPreservedInPrunedIndex = float(lineElements[-3])
            #percentageQueryProcessingCost = float(lineElements[-2])
            currentXValues.append(percentageIndexKept)
            if qualityStandardValue == 0:
                currentYValues.append(percentageTOP10DocumentResultPreservedAt10)
            elif qualityStandardValue == 2:
                currentYValues.append(percentageTOP10PostingPreservedInPrunedIndex)
            # currentYValues.append(percentageQueryProcessingCost)
    
    # final print
    print currentHeaderLine.strip()
    for index,value in enumerate(currentXValues):
        print currentXValues[index],currentYValues[index]
    print
    
    print "currentXValues:",currentXValues
    print "currentYValues:",currentYValues
    allXValueList.append(currentXValues)
    allYValueList.append(currentYValues)
    inputFileHandler.close()
    
    print "Overall big plot"
    print len(allXValueList)
    print len(allYValueList)
    #exit(1)
    
    # exit(1)
    x_AXIS_LABEL = "index kept"
    tStrP1 = x_AXIS_LABEL + " " + "versus" + " " + y_AXIS_LABEL 
    # tStrP2 = "dataset=GOV2, model=unigram"
    tStrP2 = "dataset=Gov2, model=unigram, dynamicWeight=5"
    tStrComplete = ""
    # tStrComplete = tStrP1 + ". " + tStrP2
    plt.title(tStrComplete)
    plt.xlabel(x_AXIS_LABEL)
    plt.ylabel(y_AXIS_LABEL)
    colorList = ["b","g","c","m","y","k","w","b","g","c","m","y","k","w","b","g","c","m","y","k","w"]
    
    tempList1 = ["dynamic weight == 0", "dynamic weight == 5","dynamic weight == 10"]
    tempList2 = ["dynamic weight == 0", "dynamic weight == 3","dynamic weight == 5"]
    tempList3 = ["unigram dynamic - 10","unigram dynamic - 5","unigram static - 0","bigrams-0.78-qb.static","bigrams-0.78-qb.dyn","bigrams-1-qb.static","bigrams-1-qb.dyn","bigrams-0.75-qb-gov.static","bigrams-0.75-qb-gov.dyn","bigrams-0.54-qb-static","bigrams-0.54-qb.dyn"]
    tempList4 = ["avg","1","2","3","4","5","6","7","8","9","10"]
    print "len(colorList):",len(colorList)
    print "len(markerList):",len(markerList)
    print "len(tempList1):",len(tempList1)
    print "len(tempList2):",len(tempList2)
    print "len(tempList3):",len(tempList3)
    print "len(tempList4):",len(tempList4)
    # exit(1)
    
    labelList = tempList4
    linestyleStr = "solid"
    lineWidthValue = 4
    
    for i in range(0,len(allXValueList)):
        plt.plot(allXValueList[i][1:11], allYValueList[i][1:11], \
                 color=colorList[i], linestyle=linestyleStr, linewidth=lineWidthValue, marker="o", markerfacecolor='k', markersize=markersizeValue, label=labelList[i])    
    
    plt.axis([0.0, 0.1, 0.0, 1.0])
    plt.grid(True)
    plt.legend(loc="upper left")
    
    
    plt.rcParams.update({'font.size': 18})
    fig = plt.gcf()
    fig.set_size_inches(18.5,12.5)
    fig.savefig(OFN1,format='eps', dpi=1000, papertype="letter", bbox_inches='tight')
    fig.savefig(OFN2,format='png', papertype="letter", bbox_inches='tight')
    
    print "OFN1:",OFN1
    print "OFN2:",OFN2
    
    plt.show()

def plotForCostModel(ifn):
    xValues = []
    y1Values_0 = []
    y2Values_0 = []
    y1Values_Dot1 = []
    y2Values_Dot1 = []
    y1Values_Dot3 = []
    y2Values_Dot3 = []
    y1Values_Dot5 = []
    y2Values_Dot5 = []
    y1Values_Dot7 = []
    y2Values_Dot7 = []
    y1Values_Dot9 = []
    y2Values_Dot9 = []
    y1Values_1 = []
    y2Values_1 = []
     
    ifh = open(ifn,"r")
    # ptPowTo0 d% 1734 0.69314718056
    for l in ifh.readlines():
        le = l.strip().split(" ")
        if le[0] == "ptPowTo0":
            if le[1][:-1] == "d":
                pass
            else:
                xValues.append(float(le[1][:-1])/100)
                y1Values_0.append(float(le[4]))
                y2Values_0.append(float(le[5]))
        if le[0] == "ptPowToDot1":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_Dot1.append(float(le[4]))
                y2Values_Dot1.append(float(le[5]))
                
        if le[0] == "ptPowToDot3":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_Dot3.append(float(le[4]))
                y2Values_Dot3.append(float(le[5]))

        if le[0] == "ptPowToDot5":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_Dot5.append(float(le[4]))
                y2Values_Dot5.append(float(le[5]))

        if le[0] == "ptPowToDot7":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_Dot7.append(float(le[4]))
                y2Values_Dot7.append(float(le[5]))

        if le[0] == "ptPowToDot9":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_Dot9.append(float(le[4]))
                y2Values_Dot9.append(float(le[5]))

        if le[0] == "ptPowTo1":
            if le[1][:-1] == "d":
                pass
            else:
                #xValues.append(float(le[1][:-1])/100)
                y1Values_1.append(float(le[4]))
                y2Values_1.append(float(le[5]))
        
    ifh.close()
    print "len(xValues):",len(xValues)
    print "len(y1Values_0):",len(y1Values_0)
    print "len(y1Values_Dot1):",len(y1Values_Dot1)
    print "len(y1Values_Dot3):",len(y1Values_Dot3)
    print "len(y1Values_Dot5):",len(y1Values_Dot5)
    print "len(y1Values_Dot7):",len(y1Values_Dot7)
    print "len(y1Values_Dot9):",len(y1Values_Dot9)
    print "len(y1Values_1):",len(y1Values_1)
    # exit(1)
    
    #plt.plot(xValues, y1Values_0, \
    #         color='b', linestyle='solid', marker=markerList[0], markerfacecolor='r', markersize=6, label='Pow Order - 0')

    plt.plot(xValues, y2Values_0, \
             color='b', linestyle='dashed', marker=markerList[0], markerfacecolor='r', markersize=6, label='Pow Order - 0')

    #plt.plot(xValues, y1Values_Dot1, \
    #         color='b', linestyle='solid', marker=markerList[1], markerfacecolor='r', markersize=6, label='Pow Order - 0.1')

    plt.plot(xValues, y2Values_Dot1, \
             color='b', linestyle='dashed', marker=markerList[1], markerfacecolor='r', markersize=6, label='Pow Order - 0.1')
    
    #plt.plot(xValues, y1Values_Dot3, \
    #         color='b', linestyle='solid', marker=markerList[2], markerfacecolor='r', markersize=6, label='Pow Order - 0.3')

    plt.plot(xValues, y2Values_Dot3, \
             color='b', linestyle='dashed', marker=markerList[2], markerfacecolor='r', markersize=6, label='Pow Order - 0.3')

    #plt.plot(xValues, y1Values_Dot5, \
    #         color='b', linestyle='solid', marker=markerList[3], markerfacecolor='r', markersize=6, label='Pow Order - 0.5')

    plt.plot(xValues, y2Values_Dot5, \
             color='b', linestyle='dashed', marker=markerList[3], markerfacecolor='r', markersize=6, label='Pow Order - 0.5')

    #plt.plot(xValues, y1Values_Dot7, \
    #         color='b', linestyle='solid', marker=markerList[4], markerfacecolor='r', markersize=6, label='Pow Order - 0.7')

    plt.plot(xValues, y2Values_Dot7, \
             color='b', linestyle='dashed', marker=markerList[4], markerfacecolor='r', markersize=6, label='Pow Order - 0.7')

    #plt.plot(xValues, y1Values_Dot9, \
    #         color='b', linestyle='solid', marker=markerList[5], markerfacecolor='r', markersize=6, label='Pow Order - 0.9')

    plt.plot(xValues, y2Values_Dot9, \
             color='b', linestyle='dashed', marker=markerList[5], markerfacecolor='r', markersize=6, label='Pow Order - 0.9')
    
    
    #plt.plot(xValues, y1Values_1, \
    #         color='b', linestyle='solid', marker=markerList[6], markerfacecolor='r', markersize=6, label='Pow Order - 1')

    plt.plot(xValues, y2Values_1, \
             color='b', linestyle='dashed', marker=markerList[6], markerfacecolor='r', markersize=6, label='Pow Order - 1')

    plt.xlabel("Index Kept")
    # plt.ylabel("Query Processing Cost (the SUM cost model)")
    plt.ylabel("Query Processing Cost (the LOG cost model)")
    
    plt.axis([0, 1, 0, 1])
    plt.legend(loc="upper left")
    plt.show()

def plotOverallPlots(ifn):
    inputFileHandler = open(ifn,"r")
    inputFileHandler.readline()
    allXValueList = inputFileHandler.readline().strip().split(" ")
    allYValuesList = []
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        allYValuesList.append(lineElements)

def plotTiering(fileNameList):
    print "Begins..."
    currentXPoints = []
    currentYPoints = []
    for ifn in fileNameList:
        currentXPoints = []
        currentYPoints = []
        print "ifn:",ifn
        ifh = open(ifn,"r")
        l = ifh.readline()
        costN = 0
        costD = 0
        qualityN = 0
        qualityD = 0
        beginQuality = 0
        beginCost = 0
        while l:
            le = l.strip().split(" ")
            qualityN += int(le[1])
            qualityD += int(le[3])
            costN += int(le[2])
            costD += int(le[4])
            l = ifh.readline()
        beginQuality = qualityN/qualityD
        beginCost = costN/costD
        #print "Overall:"
        #print "beginQuality:",beginQuality
        #print "beginCost:",beginCost
        #print
        print str(beginQuality),str(beginCost),"0","0.0","50%","100%","ptPowTo0",beginQuality,beginCost,"0","0"
        currentXPoints.append(beginCost)
        currentYPoints.append(beginQuality)    
        ifh.close()
        
        
        # 2ed pass
        NUM_OF_QUERIS = 4981
        currQuality = beginQuality
        currCost = beginCost
        ifh = open(ifn,"r")
        l = ifh.readline()
        numOfQueriesFallingThrough = 0  
        while l:
            le = l.strip().split(" ")
            qualityN += int(le[3]) - int(le[1])
            costN += int(le[4])
            l = ifh.readline()
            numOfQueriesFallingThrough += 1
            currQuality = qualityN/qualityD
            currCost = costN/costD
            print currQuality,currCost,numOfQueriesFallingThrough,numOfQueriesFallingThrough/NUM_OF_QUERIS,"50%","100%","ptPowTo0",beginQuality,beginCost,currQuality-beginQuality,currCost-beginCost
            currentXPoints.append(currCost)
            currentYPoints.append(currQuality)
        print
        ifh.close()
        print "Overall:"
        print "len(currentXPoints):",len(currentXPoints)
        print "len(currentYPoints):",len(currentYPoints)
        plt.plot(currentXPoints,currentYPoints)
    
    plt.axis([0, 1.1, 0, 1.1])
    plt.grid(True)
    plt.ylabel("quality")
    plt.xlabel("cost")
    plt.show()
    print "Ends."

print "Program Begins..."
# Updated by Wei on 2014/10/06
#inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/results_UPP_5_unigram_20141006"
#dataForChartPlottingFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/qualityControl_dataPointsForPlotting_gov2_tail5K_20141006.txt"
#doRegression(inputFileName,dataForChartPlottingFileName)
#exit(1)

'''
# Updated by Wei on 2014/08/26
# tiering
fileNameList = []
# 50%
ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/final/ptPowTo0_step3_final_15"
# 60%
ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/final/ptPowTo0_step3_final_16"
# 70%
ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/final/ptPowTo0_step3_final_17"
# 80%
ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/final/ptPowTo0_step3_final_18"
# 90%
ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/final/ptPowTo0_step3_final_19"

fileNameList.append(ifn1)
fileNameList.append(ifn2)
fileNameList.append(ifn3)
fileNameList.append(ifn4)
fileNameList.append(ifn5)
plotTiering(fileNameList)
exit(1)
'''

# Updated by Wei on 2014/07/22
# plot for bigrams, just make it work! Don't miss one big contribution and findings.
# Using bigrams from gov2
#inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/bigramResultsFromJuan_20140722.txt"
#plotBigrams(inputFileName)
#exit(0)

# Updated by Wei on 2014/07/21
#ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/costModelComparizon_20140721"
#plotForCostModel(ifn)
#exit(1)

# Updated by Wei on 2014/07/25
# currently NOT under construction ! Let Juan do this maybe.
# plot result kept
# ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingKept_20140725.csv"
# ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultKept_20140725.csv"
# ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/pAt10_20140725.csv"
# plotOverallPlots(ifn)
# exit(1)

# Updated by Wei on 2014/07/20
# 0 for result kept
# 2 for posting kept
# qualityStandardValue = 2
# plot for varying the dynamic weights using the gov2 dataset.
# Using gov2 (working)
# inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dataResultsForPlottingVaryingAlpha_20140719_GOV2.txt"
# Using clueweb09B
# (NOT USING) inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dataResultsForPlottingVaryingAlpha_20140719_Clueweb09B.txt"
# plotVaryingDynamicWeight(inputFileName,qualityStandardValue)
# exit(0)

# NOT GOING to use it anymore since 20140725
# Updated by Wei on 2014/07/20
# 0 for result kept
# 2 for posting kept
#qualityStandardValue = 0
# plot for query length breaking down
#inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dataResultsForPlottingBreakingDownQueryLength_20140720_GOV2.txt"
#plotBreakingDownQueryLength(inputFileName,qualityStandardValue)
#exit(0)

# Updated by Wei on 2014/07/26
# step1:
# inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/quality_control_gov2_tail5K_ptChangedFromJuan_20140721_otherMethodsAdded_logCostModelAdded.csv"
# inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/quality_control_gov2_tail5K_ptChangedFromJuan_20140723_otherMethodsAdded_logCostModelAdded.csv"
# inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/results_dynamicWeight_0_5_unigram_20140723"
# current working
# inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/results_dynamicWeight_0_5_unigram_20140723"
# current testing
#inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/results_dynamicWeight_0_5_unigram_20140724"
#dataForChartPlottingFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/qualityControl_dataPointsForPlotting_gov2_tail5K_20140723_otherMethodsAdded_logCostModelAdded.txt"
dataForChartPlottingFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/qualityControl_dataPointsForPlotting_gov2_tail5K_20141006.txt"
# (working and important) dataForChartPlottingFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dataPointsForPlotting_gov2_tail5K_dynamicWeight_0_5_unigram_20140724"
# dataForChartPlottingFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dataPointsForPlotting_gov2_tail5K_dynamicWeight_0_5_unigram_20140724"
#doRegression(inputFileName,dataForChartPlottingFileName)
#exit(1)

# step2:
# for gov2 tail5K
# arg1: inputFileName
# arg2: quality standard
    # 0 for result kept
    # 1 for posting retained@10 (NOT USED currently)
    # 2 for posting retained in pruned index
# arg3: cost model value
    # 0 for sum cost model
    # 1 for log cost model
    # 2 for ms cost model
oneBigPlotForGOV2Tail5K_QPC_indexSize_tradeoff(dataForChartPlottingFileName,0,0)
print "Program Ends."
exit(0)



